{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Machine Translator with Seq2seq: German to English"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "# These are all the modules we'll be using later. Make sure you can import them\n",
    "# before proceeding further.\n",
    "%matplotlib inline\n",
    "from __future__ import print_function\n",
    "import collections\n",
    "import math\n",
    "import numpy as np\n",
    "import os\n",
    "import random\n",
    "import tensorflow as tf\n",
    "import zipfile\n",
    "from matplotlib import pylab\n",
    "from six.moves import range\n",
    "from six.moves.urllib.request import urlretrieve\n",
    "import tensorflow as tf\n",
    "from PIL import Image\n",
    "from collections import Counter\n",
    "import csv\n",
    "# Seq2Seq Items\n",
    "import tensorflow.contrib.seq2seq as seq2seq\n",
    "from tensorflow.python.ops.rnn_cell import LSTMCell\n",
    "from tensorflow.python.ops.rnn_cell import MultiRNNCell\n",
    "from tensorflow.contrib.seq2seq.python.ops import attention_wrapper\n",
    "from tensorflow.python.layers.core import Dense"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining Hyperparameters\n",
    "We define main hyperparameters required for the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vocab_size= 50000\n",
    "num_units = 128\n",
    "input_size = 128\n",
    "batch_size = 16\n",
    "source_sequence_length=40\n",
    "target_sequence_length=60\n",
    "decoder_type = 'basic' # could be basic or attention"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Preprocessing (Copied from Chapter 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Data \n",
    "\n",
    "First, download the data from this [page](https://nlp.stanford.edu/projects/nmt/) and place them in the ch10 folder. You do not need to do anything if you have already run the exercises in `ch10` folder. The required files are:\n",
    "\n",
    "* File containing German sentences: [`train.de`](https://nlp.stanford.edu/projects/nmt/data/wmt14.en-de/train.de)\n",
    "* File containing English sentences: [`train.en`](https://nlp.stanford.edu/projects/nmt/data/wmt14.en-de/train.en)\n",
    "* File containing German vocabulary: [`vocab.50K.de`](https://nlp.stanford.edu/projects/nmt/data/wmt14.en-de/vocab.50K.de)\n",
    "* File containing English vocabulary: [`vocab.50K.en`](https://nlp.stanford.edu/projects/nmt/data/wmt14.en-de/vocab.50K.en)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading Vocabulary\n",
    "\n",
    "First we build the vocabulary dictionaries for both the source (German) and target (English) languages. The vocabularies are found in the `vocab.50K.de` (German) and `vocab.50K.en` files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source\n",
      "\t [('veraltete', 20446), ('Norditalien', 45398), ('89', 10378), ('WD', 19593), ('beiderseitigen', 24051), ('Filmfestivals', 49249), ('Wahlkommission', 34930), ('piscine', 14869), ('berechtigterweise', 34797), ('Budgetierung', 34984)]\n",
      "\t [(0, '<unk>'), (1, '<s>'), (2, '</s>'), (3, ','), (4, '.'), (5, 'die'), (6, 'der'), (7, 'und'), (8, 'in'), (9, 'zu')]\n",
      "\t Vocabulary size:  50000\n",
      "Target\n",
      "\t [('younger', 5563), ('spade', 21549), ('89', 9143), ('CF', 19570), ('decentralised', 10082), ('piscine', 23926), ('Creek', 19483), ('JP', 36526), ('Single', 4328), ('forget', 1823)]\n",
      "\t [(0, '<unk>'), (1, '<s>'), (2, '</s>'), (3, 'the'), (4, ','), (5, '.'), (6, 'of'), (7, 'and'), (8, 'to'), (9, 'in')]\n",
      "\t Vocabulary size:  50000\n"
     ]
    }
   ],
   "source": [
    "# ==========================================\n",
    "# Building source language vocabulary\n",
    "\n",
    "# Contains word string -> ID mapping\n",
    "src_dictionary = dict()\n",
    "\n",
    "# Read the vocabulary file\n",
    "with open(os.path.join('..','ch10','vocab.50K.de'), encoding='utf-8') as f:\n",
    "    # Read and store every line\n",
    "    for line in f:\n",
    "        #we are discarding last char as it is new line char\n",
    "        src_dictionary[line[:-1]] = len(src_dictionary)\n",
    "\n",
    "# Build a reverse dictionary with the mapping ID -> word string\n",
    "src_reverse_dictionary = dict(zip(src_dictionary.values(),src_dictionary.keys()))\n",
    "\n",
    "# Print some of the words in the dictionary\n",
    "print('Source')\n",
    "print('\\t',list(src_dictionary.items())[:10])\n",
    "print('\\t',list(src_reverse_dictionary.items())[:10])\n",
    "print('\\t','Vocabulary size: ', len(src_dictionary))\n",
    "\n",
    "# ==========================================\n",
    "# Building source language vocabulary\n",
    "\n",
    "# Contains word string -> ID mapping\n",
    "tgt_dictionary = dict()\n",
    "\n",
    "# Read the vocabulary file\n",
    "with open(os.path.join('..','ch10','vocab.50K.en'), encoding='utf-8') as f:\n",
    "    # Read and store every line\n",
    "    for line in f:\n",
    "        #we are discarding last char as it is new line char\n",
    "        tgt_dictionary[line[:-1]] = len(tgt_dictionary)\n",
    "\n",
    "# Build a reverse dictionary with the mapping ID -> word string\n",
    "tgt_reverse_dictionary = dict(zip(tgt_dictionary.values(),tgt_dictionary.keys()))\n",
    "\n",
    "# Print some of the words in the dictionary\n",
    "print('Target')\n",
    "print('\\t',list(tgt_dictionary.items())[:10])\n",
    "print('\\t',list(tgt_reverse_dictionary.items())[:10])\n",
    "print('\\t','Vocabulary size: ', len(tgt_dictionary))\n",
    "\n",
    "# Each language has 50000 words\n",
    "vocabulary_size = 50000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading Training and Testing Data\n",
    "\n",
    "Here we load the data in the `train.de` and `train.en` files. And split the data in the files into two sets; training and testing data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sample translations (250000)\n",
      "( 0 ) DE:  Hier erfahren Sie , wie Sie Creative Suite 2 und Creative Suite 3 am besten zusammen mit QuarkXPress nutzen können .\n",
      "\n",
      "( 0 ) EN:  Here , you ’ ll find out how Creative Suite users can get the best possible interaction with QuarkXPress .\n",
      "\n",
      "( 10000 ) DE:  Für die sehr günstigen Wochen- und Monatskarten ( 1 Monat ca.\n",
      "\n",
      "( 10000 ) EN:  It is THE trendy area of Marseille .\n",
      "\n",
      "( 20000 ) DE:  Freuen Sie sich auf die romantische Atmosphäre in den Zimmern und Apartments .\n",
      "\n",
      "( 20000 ) EN:  Enjoy the romantic atmosphere of one of the guest rooms or apartments .\n",
      "\n",
      "( 30000 ) DE:  Zu zwiespältig sind Dr. Gutherzens Erfahrungen aus frühen Studententagen verlaufen , in denen er sich in die Gefielde von durch Heidegger geprägten Autor / innen begeben hat und dort ständig mit strengem Blick darauf verwiesen wurde , er habe bestimmte Theorieressourcen und Gedankengebäude einfach noch nicht gründlich genug verstanden und könne deshalb nicht begreifen , warum seine Einwände zu bestimmten Texten und Diskursen nicht stichhaltig seien .\n",
      "\n",
      "( 30000 ) EN:  This vagueness lends itself to an idealisation of violence , formulated in concepts of &quot; assault &quot; against imaginary authorities or enthusiastic notions of &quot; blissful traumatic knowledge &quot; .\n",
      "\n",
      "( 40000 ) DE:  Sie veranlassen den untergeordneten Prozess , sich während seiner gesamten Lebensdauer lediglich einmal mit dem SQL ##AT##-##AT## Server zu verbinden , anstatt bei jedem Aufruf einer Seite , die eine Verbindung benötigt .\n",
      "\n",
      "( 40000 ) EN:  They cause the child process to simply connect only once for its entire lifespan , instead of every time it processes a page that requires connecting to the SQL server .\n",
      "\n",
      "( 50000 ) DE:  Je intensiver man dabei bleibt , desto bessere Ergebnisse erzielt man .\n",
      "\n",
      "( 50000 ) EN:  The more intensively you do them , the better the results .\n",
      "\n",
      "( 60000 ) DE:  In allen Zimmern ist Digitalfernsehen und Internetzugang für sowohl Geschäftsreisende als auch Urlauber erhältlich .\n",
      "\n",
      "( 60000 ) EN:  All rooms offer digital TV and Internet access appealing to both corporate and leisure guests .\n",
      "\n",
      "( 70000 ) DE:  Bitte beachten Sie , dass Ihr Check ##AT##-##AT## in ##AT##-##AT## Code nicht mit der Buchungsnummer identisch ist .\n",
      "\n",
      "( 70000 ) EN:  Please note that the check ##AT##-##AT## in number and your reservation number are not the same .\n",
      "\n",
      "( 80000 ) DE:  Auch die Art , wie man einen eigenen Weißabgleich vornehmen kann , darf angepasst werden .\n",
      "\n",
      "( 80000 ) EN:  Another thing that should be reassessed is the way in which the user creates his own white balance .\n",
      "\n",
      "( 90000 ) DE:  Weitere Supportoptionen ( http : / / support.microsoft.com / contactus ) : Stellen Sie Ihre Fragen im Web , wenden Sie sich an Microsoft Support Services , oder teilen Sie uns Ihre Meinung mit .\n",
      "\n",
      "( 90000 ) EN:  Other Support Options ( http : / / support.microsoft.com / default.aspx ? pr = csshome ) : Use the Web to ask a question , contact Microsoft Customer Support Services , or provide feedback .\n",
      "\n",
      "( 100000 ) DE:  Ik vond het geen 4 ##STAR## waard . Het appartement oogde erg schroezelig en gedateerd , personeel sprak erg gebrekkig Engels .\n",
      "\n",
      "( 100000 ) EN:  Trousse d &apos;information manquante et devrait inclure une carte du site ainsi que des services et activités sur les lieu ou dans la commune ainsi les attraits touristiques de la région .\n",
      "\n",
      "( 110000 ) DE:  Dieses Bild spiegelt sich in Ihrem Unternehmen und Ihren Produkten wieder .\n",
      "\n",
      "( 110000 ) EN:  This image reflects on your company and products .\n",
      "\n",
      "( 120000 ) DE:  Alle Zimmer sind mit Digital ##AT##-##AT## TV und DVD und kostenlosem Breitbandanschluss sowie Direktwahltelefon ausgestattet .\n",
      "\n",
      "( 120000 ) EN:  Our rooms include a romantic four ##AT##-##AT## poster and two easy access ground floor rooms . All rooms are equipped to hotel standards with Digital TV and DVD , free broadband connections and free local and national direct dial phones .\n",
      "\n",
      "( 130000 ) DE:  Nothing if im quite honet . I wouldnt stay here again or recommend it to anyone i know .\n",
      "\n",
      "( 130000 ) EN:  the room was basic but spacoius and clean , the staff were friendly and helpful , the food was tasty , all in all , lovely place to stay !\n",
      "\n",
      "( 140000 ) DE:  Es gibt 4 verschiedene Möglichkeiten , Cannon Blast zu Ihrem Blog oder Ihrer Website hinzuzufügen .\n",
      "\n",
      "( 140000 ) EN:  There are 4 different ways of posting Cannon Blast to your blog or website .\n",
      "\n",
      "( 150000 ) DE:  Wenn die Buchung vor 14 : 00 Uhr 3 , Tage vor dem geplanten Anreisetag storniert wird , fällt keine Stornierungsgebühr an .\n",
      "\n",
      "( 150000 ) EN:  There will be no cancellation charge if a booking is cancelled before 14 : 00 3 days before your date of arrival .\n",
      "\n",
      "( 160000 ) DE:  Im geräumigen Hotelrestaurant Al Caminetto kosten Sie Gerichte aus Mailand und aus aller Welt .\n",
      "\n",
      "( 160000 ) EN:  The hotel ’ s restaurant , Al Caminetto , serves Milanese and international cuisine .\n",
      "\n",
      "( 170000 ) DE:  Während der 60 &apos; er Jahre gab es viele Regisseure die in die Wüste von Ameria zogen um , mit der ...\n",
      "\n",
      "( 170000 ) EN:  During the 1960s , numerous movie directors chose Almeria &apos;s desert ##AT##-##AT## like landscape to film some of ...\n",
      "\n",
      "( 180000 ) DE:  Tikje krappe kamer voor het aanwezige meubilair en de lift is absoluut niet meer van deze tijd : veel te klein .\n",
      "\n",
      "( 180000 ) EN:  Chambre minuscule , rien à voir avec les photos présentées , SDB &quot; vieillotte &quot; . Absence totale d &apos;insonorisation : l &apos;intimité de vos voisins de chambre en direct ....... Séjour écourté ......\n",
      "\n",
      "( 190000 ) DE:  Das Großunternehmen sieht sich einfach die Produkte des kleinen Unternehmens an und unterstellt so viele Patentverletzungen , wie es nur geht .\n",
      "\n",
      "( 190000 ) EN:  The large corporation will look at the products of the small company and bring up as many patent infringement assertions as possible .\n",
      "\n",
      "( 200000 ) DE:  Wochentags bis 22 Uhr , Samstags bis 18 Uhr geöffnet . Sehr sympathische Atmosphäre .\n",
      "\n",
      "( 200000 ) EN:  This is an interactive multimedia tour ( choice of languages ) through Weimar &apos;s history from prehistoric times to the present .\n",
      "\n",
      "( 210000 ) DE:  Wann möchten Sie im Entrecercas übernachten ?\n",
      "\n",
      "( 210000 ) EN:  When would you like to stay at the Entrecercas ?\n",
      "\n",
      "( 220000 ) DE:  In der ordentlichen Sitzung am 22. September 2008 befasste sich der Aufsichtsrat mit strategischen Themen aus den einzelnen Geschäftsbereichen wie der Positionierung des Kassamarktes im Wettbewerb mit außerbörslichen Handelsplattformen , den Innovationen im Derivatesegment und verschiedenen Aktivitäten im Nachhandelsbereich .\n",
      "\n",
      "( 220000 ) EN:  At the regular meeting on 22 September 2008 , the Supervisory Board dealt with strategic issues from the various business areas , such as the positioning of the cash market in competition with OTC trading platforms , innovation in the derivatives segment and various post ##AT##-##AT## trading activities .\n",
      "\n",
      "( 230000 ) DE:  Ich hatte keine Sekunde zum Entspannen .\n",
      "\n",
      "( 230000 ) EN:  I never had even one second to relax .\n",
      "\n",
      "( 240000 ) DE:  Das Englisch sprechende Personal steht Ihnen mit Rat und Tat zur Seite , informiert über Sehenswürdigkeiten und arrangiert Ihren Transfer .\n",
      "\n",
      "( 240000 ) EN:  The English ##AT##-##AT## speaking staff are always on hand to make your stay special .\n",
      "\n",
      "Sample test translations (100)\n",
      "DE:  Heute verstehen sich QuarkXPress ® 8 , Photoshop ® und Illustrator ® besser als jemals zuvor . Dank HTML und CSS ­ können Anwender von QuarkXPress inzwischen alle Medien bedienen , und das unabhängig von Anwendungen der Adobe ® Creative Suite ® wie Adobe Flash ® ( SWF ) und Adobe Dreamweaver ® .\n",
      "\n",
      "EN:  Today , QuarkXPress ® 8 has tighter integration with Photoshop ® and Illustrator ® than ever before , and through standards like HTML and CSS , QuarkXPress users can publish across media both independently and alongside Adobe ® Creative Suite ® applications like Adobe Flash ® ( SWF ) and Adobe Dreamweaver ® .\n",
      "\n",
      "DE:  Das Hotel Opera befindet sich in der Nähe des Royal Theatre , Kongens Nytorv , &apos; Stroget &apos; und Nyhavn .\n",
      "\n",
      "EN:  Hotel Opera is situated near The Royal Theatre , Kongens Nytorv , &quot; Strøget &quot; and fascinating Nyhavn .\n",
      "\n",
      "DE:  Es existieren Busverbindungen in nahezu jeden Ort der Provence ( eventuell mit Umsteigen in Aix ##AT##-##AT## en ##AT##-##AT## Provence ) , allerdings sollte beachtet werden , dass die letzten Busse abends ca. um 19 Uhr fahren .\n",
      "\n",
      "EN:  As always in France those highways are expensive but practical , comfortable and fast .\n",
      "\n",
      "DE:  15. einem Dritten bei dem Verstoss gegen eine dieser Regeln zu helfen .\n",
      "\n",
      "EN:  15. assist any third party in engaging in any activity prohibited by these Terms .\n",
      "\n",
      "DE:  Es war staubig , das Bad schmutzig . Sogar die Beleuchtung an der Wand im Flur ( Seitengebäude ) war richtig verstaubt .\n",
      "\n",
      "EN:  It was rather old fashioned in the decoration .\n",
      "\n",
      "DE:  Die Bewohner des Nordens sind ein buntes Völkergemisch aus den verschiedensten Bergstämmen und den Nord ##AT##-##AT## Thais oder kon mueang ; die traditionell in den fruchtbaren Tiefebenen Nordthailands siedeln . In vielerlei Hinsicht halten sich die Nord Thais für die &quot; wahren &quot; Thais , die die Thai ##AT##-##AT## Kultur noch am besten über die Zeit gerettet haben .\n",
      "\n",
      "EN:  From Pratu Chiang Mai market , songthaews also travel to Hang Dong ( 20 baht ) and San Patong , south ##AT##-##AT## west of Chiang Mai .\n",
      "\n",
      "DE:  Auch ist , so denkt Dr. Gutherz , bereits die erste Seite sehr viel versprechend , da sie eine Definition des klinischen Psychotrauma ##AT##-##AT## Begriffes enthält , der er gänzlich zustimmen kann .\n",
      "\n",
      "EN:  At the rhetorical climax of this summary , Dr Goodheart comes across some sentences expressed with great pathos .\n",
      "\n",
      "DE:  Das Cleddau Bridge Hotel ist der ideale Platz um zu entspannen oder geschäftlich zu reisen .\n",
      "\n",
      "EN:  Cleddau Bridge hotel is the ideal place for those who want a relaxing holiday or who travel for business .\n",
      "\n",
      "DE:  Bei einer digitalen Bildkette wird das Intensitätssignal für jedes Pixel ohne analoge Zwischenschritte direkt in der Detektoreinheit digitalisiert , d.h. in Zahlen umgewandelt .\n",
      "\n",
      "EN:  A digital image chain is an image chain that is equipped with a digital detector instead of an analogue one .\n",
      "\n",
      "DE:  Sehr freundliche Auszubildende an der Rezeption , die sehr bemüht noch einen Flug für mich gebucht hat .\n",
      "\n",
      "EN:  First of all I did not like the price ... the next day I went to Milano to a 4 star Hotel for 10 Euro less and super service .. I had a problem with my Internetconnection and the Hotel Maritim did not react right .\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Contains the training sentences\n",
    "source_sent = [] # Input\n",
    "target_sent = [] # Output\n",
    "\n",
    "# Contains the testing sentences\n",
    "test_source_sent = [] # Input\n",
    "test_target_sent = [] # Output\n",
    "\n",
    "# We grab around 100 lines of data that are interleaved \n",
    "# in the first 50000 sentences\n",
    "test_indices = [l_i for l_i in range(50,50001,500)]\n",
    "\n",
    "# Read the source data file and read the first 250,000 lines (except first 50)\n",
    "with open(os.path.join('..','ch10','train.de'), encoding='utf-8') as f:\n",
    "    for l_i, line in enumerate(f):\n",
    "        # discarding first 50 translations as there was some\n",
    "        # english to english mappings found in the first few lines. which are wrong\n",
    "        if l_i<50:\n",
    "            continue\n",
    "        \n",
    "        if len(source_sent)<250000 and l_i not in test_indices:\n",
    "            source_sent.append(line)\n",
    "        elif l_i in test_indices:\n",
    "            test_source_sent.append(line)\n",
    "        \n",
    "# Read the target data file and read the first 250,000 lines (except first 50)            \n",
    "with open(os.path.join('..','ch10','train.en'), encoding='utf-8') as f:\n",
    "    for l_i, line in enumerate(f):\n",
    "        # discarding first 50 translations as there was some\n",
    "        # english to english mappings found in the first few lines. which are wrong\n",
    "        if l_i<50:\n",
    "            continue\n",
    "        \n",
    "        if len(target_sent)<250000 and l_i not in test_indices:\n",
    "            target_sent.append(line)\n",
    "        elif l_i in test_indices:\n",
    "            test_target_sent.append(line)\n",
    "        \n",
    "# Make sure we extracted same number of both extracted source and target sentences         \n",
    "assert len(source_sent)==len(target_sent),'Source: %d, Target: %d'%(len(source_sent),len(target_sent))\n",
    "\n",
    "# Print some source sentences\n",
    "print('Sample translations (%d)'%len(source_sent))\n",
    "for i in range(0,250000,10000):\n",
    "    print('(',i,') DE: ', source_sent[i])\n",
    "    print('(',i,') EN: ', target_sent[i])\n",
    "\n",
    "# Print some target sentences\n",
    "print('Sample test translations (%d)'%len(test_source_sent))\n",
    "for i in range(0,100,10):\n",
    "    print('DE: ', test_source_sent[i])\n",
    "    print('EN: ', test_target_sent[i])\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing text\n",
    "Here we preprocess the text by replacing words not found in the dictionary with `<unk>` as well as remove punctuation marks (`.`,`,`) and new-line characters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(Source) Sentence mean length:  26.244692\n",
      "(Source) Sentence stddev length:  13.854376414156501\n",
      "(Target) Sentence mean length:  28.275308\n",
      "(Target) Sentence stddev length:  14.925498769057468\n",
      "(Test-Source) Sentence mean length:  26.61\n",
      "(Test-Source) Sentence stddev length:  14.800604717375572\n",
      "(Test-Target) Sentence mean length:  29.08\n",
      "(Test-Target) Sentence stddev length:  16.19424589167399\n"
     ]
    }
   ],
   "source": [
    "# Keep track of how many unknown words were encountered\n",
    "src_unk_count, tgt_unk_count = 0, 0\n",
    "\n",
    "def split_to_tokens(sent,is_source):\n",
    "    '''\n",
    "    This function takes in a sentence (source or target)\n",
    "    and preprocess the sentency with various steps (e.g. removing punctuation)\n",
    "    '''\n",
    "    \n",
    "    global src_unk_count, tgt_unk_count\n",
    "\n",
    "    # Remove punctuation and new-line chars\n",
    "    sent = sent.replace(',',' ,')\n",
    "    sent = sent.replace('.',' .')\n",
    "    sent = sent.replace('\\n',' ') \n",
    "    \n",
    "    sent_toks = sent.split(' ')\n",
    "    for t_i, tok in enumerate(sent_toks):\n",
    "        if is_source:\n",
    "            # src_dictionary contain the word -> word ID mapping for source vocabulary\n",
    "            if tok not in src_dictionary.keys():\n",
    "                if not len(tok.strip())==0:\n",
    "                    sent_toks[t_i] = '<unk>'\n",
    "                    src_unk_count += 1\n",
    "        else:\n",
    "            # tgt_dictionary contain the word -> word ID mapping for target vocabulary\n",
    "            if tok not in tgt_dictionary.keys():\n",
    "                if not len(tok.strip())==0:\n",
    "                    sent_toks[t_i] = '<unk>'\n",
    "                    #print(tok)\n",
    "                    tgt_unk_count += 1\n",
    "    return sent_toks\n",
    "\n",
    "# Let us first look at some statistics of the sentences\n",
    "# Train - source data\n",
    "source_len = []\n",
    "source_mean, source_std = 0,0\n",
    "for sent in source_sent:\n",
    "    source_len.append(len(split_to_tokens(sent,True)))\n",
    "\n",
    "print('(Source) Sentence mean length: ', np.mean(source_len))\n",
    "print('(Source) Sentence stddev length: ', np.std(source_len))\n",
    "\n",
    "# Let us first look at some statistics of the sentences\n",
    "# Train - target data\n",
    "target_len = []\n",
    "for sent in target_sent:\n",
    "    target_len.append(len(split_to_tokens(sent,False)))\n",
    "\n",
    "print('(Target) Sentence mean length: ', np.mean(target_len))\n",
    "print('(Target) Sentence stddev length: ', np.std(target_len))\n",
    "\n",
    "# Let us first look at some statistics of the sentences\n",
    "# Test - source data\n",
    "test_source_len = []\n",
    "for sent in test_source_sent:\n",
    "    test_source_len.append(len(split_to_tokens(sent, True)))\n",
    "    \n",
    "print('(Test-Source) Sentence mean length: ', np.mean(test_source_len))\n",
    "print('(Test-Source) Sentence stddev length: ', np.std(test_source_len))\n",
    "\n",
    "# Let us first look at some statistics of the sentences\n",
    "# Test - target data\n",
    "test_target_len = []\n",
    "test_tgt_mean, test_tgt_std = 0,0\n",
    "for sent in test_target_sent:\n",
    "    test_target_len.append(len(split_to_tokens(sent, False)))\n",
    "    \n",
    "print('(Test-Target) Sentence mean length: ', np.mean(test_target_len))\n",
    "print('(Test-Target) Sentence stddev length: ', np.std(test_target_len))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Making training and testing data fixed length\n",
    "\n",
    "Here we get all the source sentences and target sentences to a fixed length. This is, so that we can process the sentences as batches."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing Training Data ...\n",
      "\n",
      "Unk counts Src: 464223, Tgt: 214783\n",
      "Sentences  250000\n",
      "\t Done processing training data \n",
      "\n",
      "Samples from training data\n",
      "\t ['<s>', 'Hier', 'erfahren', 'Sie', ',', 'wie', 'Sie', 'Creative', 'Suite', '2', 'und', 'Creative', 'Suite', '3', 'am', 'besten', 'zusammen', 'mit', 'QuarkXPress', 'nutzen', 'können', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'Here', ',', 'you', '’', 'll', 'find', 'out', 'how', 'Creative', 'Suite', 'users', 'can', 'get', 'the', 'best', 'possible', 'interaction', 'with', 'QuarkXPress', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'Sie', 'werden', 'überrascht', 'sein', ',', 'wie', 'einfach', 'sich', 'mit', 'Quark', 'das', 'volle', 'Potenzial', 'Ihrer', 'Design', '##AT##-##AT##', 'Software', 'erschließen', 'lässt', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'You', '’', 'll', 'be', 'surprised', 'how', 'easy', 'Quark', 'has', 'made', 'it', 'to', 'unlock', 'the', 'full', 'potential', 'of', 'all', 'your', 'design', 'software', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'Häufig', 'wird', 'die', 'Meinung', 'vertreten', ',', 'dass', 'QuarkXPress', '8', 'von', 'allen', 'heute', 'verfügbaren', 'Layout', '##AT##-##AT##', 'Programmen', 'die', 'beste', 'Integration', 'mit', 'Photoshop', 'über', 'das', 'PSD', '##AT##-##AT##', 'Dateiformat', 'bietet', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'QuarkXPress', '8', 'is', 'considered', 'by', 'many', 'to', 'have', 'the', 'best', 'integration', 'with', 'Photoshop', '’', 's', 'PSD', 'file', 'format', 'of', 'any', 'layout', 'tool', 'available', 'today', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'In', 'diesem', 'Abschnitt', 'erläutern', 'wir', ',', 'wann', 'Sie', 'für', 'Ihre', 'Bilder', 'das', 'PSD', '##AT##-##AT##', 'Format', 'verwenden', 'sollten', 'und', 'wie', 'Sie', 'es', 'für', 'Ihre', 'Bilder', 'optimal', 'nutzen', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'In', 'this', 'section', 'we', '’', 'll', 'explain', 'when', 'you', 'should', 'use', 'the', 'PSD', 'format', 'for', 'your', 'images', 'and', 'how', 'to', 'get', 'the', 'most', 'out', 'of', 'them', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'Angenommen', 'Sie', 'haben', 'verschiedene', 'Ebenen', 'in', 'Ihrer', 'PSD', '##AT##-##AT##', 'Datei', 'mit', 'verschiedenen', 'Darstellungen', 'eines', 'Produkts', ',', 'die', 'je', 'nach', 'Verwendungszweck', 'ausgewählt', 'werden', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'For', 'example', ',', 'you', 'may', 'have', 'multiple', 'layers', 'in', 'your', 'PSD', 'with', 'different', 'product', 'shots', ',', 'which', 'will', 'vary', 'from', 'publication', 'to', 'publication', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'Wenn', 'Sie', 'mit', 'PSD', 'arbeiten', ',', 'können', 'Sie', 'diese', 'Ebenen', 'in', 'QuarkXPress', 'ein-', 'oder', 'ausschalten', ',', 'ohne', 'für', 'jede', 'Veröffentlichung', 'eine', 'eigene', 'TIFF', '##AT##-##AT##', 'Datei', 'generieren', 'zu', 'müssen', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'If', 'you', 'use', 'PSD', ',', 'you', 'can', 'switch', 'those', 'layers', 'on', 'or', 'off', 'in', 'QuarkXPress', 'without', 'having', 'to', 'save', 'a', 'separate', 'TIFF', 'for', 'each', 'publication', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'Eine', 'andere', 'mögliche', 'Frage', 'für', 'die', 'Entscheidung', 'zwischen', 'PSD', 'und', 'TIFF', 'ist', ':', '„', 'Muss', 'ich', 'für', 'dieses', 'Bild', 'eine', '<unk>', 'verwenden', '?', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'Another', 'question', 'that', 'might', 'tip', 'you', 'in', 'favor', 'of', 'PSD', 'is', ',', '&quot;', 'Do', 'I', 'need', 'to', 'use', 'a', 'spot', 'color', 'with', 'this', 'image', '?', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', '&quot;', 'In', 'den', 'meisten', '<unk>', 'sind', '<unk>', 'oft', 'problematisch', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', '&quot;', 'Using', 'spot', 'colors', 'in', 'most', 'image', 'formats', 'is', 'often', 'complicated', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'Da', 'QuarkXPress', 'jedoch', 'PSD', '##AT##-##AT##', 'Kanäle', 'unterstützt', ',', 'geht', 'es', 'mit', 'PSD', 'einfacher', 'und', 'flexibler', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'However', ',', 'because', 'of', 'the', 'way', 'QuarkXPress', 'supports', 'PSD', 'channels', ',', 'it', '’', 's', 'simpler', 'and', 'more', 'flexible', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'Erstellen', 'Sie', 'einen', 'Rahmen', 'und', 'gehen', 'Sie', 'dann', 'auf', 'Datei', '&gt;', 'Importieren', '.', '.', '.', 'oder', 'ziehen', 'Sie', 'das', 'Bild', 'einfach', 'per', 'Drag', '&amp;', 'Drop', 'von', 'Ihrem', 'Desktop', ',', 'aus', 'dem', 'Finder', 'oder', 'einer', 'Anwendung', 'wie', 'Adobe', 'Bridge', '<unk>', '–']\n",
      "\t ['<s>', 'Bringing', 'the', 'PSD', 'files', 'into', 'QuarkXPress', 'is', 'the', 'same', 'as', 'any', 'other', 'image', '.', 'Create', 'a', 'Box', 'and', 'then', 'use', 'File', '&gt;', 'Import', '.', '.', '.', 'or', 'simply', 'drag', 'and', 'drop', 'the', 'image', 'from', 'your', 'desktop', ',', 'Finder', 'or', 'an', 'application', 'like', 'Adobe', 'Bridge', '®', 'with', 'or', 'without', 'creating', 'a', 'box', 'first', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\n",
      "\tSentences  250000\n",
      "Processing testing data ....\n",
      "\n",
      "Unk counts Tgt: 212, Tgt: 107\n",
      "Done processing testing data ....\n",
      "\n",
      "Samples from training data\n",
      "\t ['<s>', 'Heute', 'verstehen', 'sich', 'QuarkXPress', '®', '8', ',', 'Photoshop', '®', 'und', 'Illustrator', '®', 'besser', 'als', 'jemals', 'zuvor', '.', 'Dank', 'HTML', 'und', 'CSS', '\\xad', 'können', 'Anwender', 'von', 'QuarkXPress', 'inzwischen', 'alle', 'Medien', 'bedienen', ',', 'und', 'das', 'unabhängig', 'von', 'Anwendungen', 'der', 'Adobe', '®', 'Creative']\n",
      "\t ['<s>', 'Today', ',', 'QuarkXPress', '®', '8', 'has', 'tighter', 'integration', 'with', 'Photoshop', '®', 'and', 'Illustrator', '®', 'than', 'ever', 'before', ',', 'and', 'through', 'standards', 'like', 'HTML', 'and', 'CSS', ',', 'QuarkXPress', 'users', 'can', 'publish', 'across', 'media', 'both', 'independently', 'and', 'alongside', 'Adobe', '®', 'Creative', 'Suite', '®', 'applications', 'like', 'Adobe', 'Flash', '®', '(', 'SWF', ')', 'and', 'Adobe', 'Dreamweaver', '®', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'Je', 'mehr', 'Zeit', 'wir', 'mit', 'Gilad', 'und', 'dem', 'Rest', 'des', 'Teams', 'in', 'Israel', 'verbracht', 'haben', '(', 'um', 'nicht', 'den', 'lauten', 'Hahn', 'zu', 'erwähnen', 'der', '<unk>', 'bei', 'denen', 'über', 'den', 'Campus', '<unk>', ')', 'desto', 'überzeugter', 'waren', 'wir', '–', 'zusammen', 'können', 'wir']\n",
      "\t ['<s>', 'The', 'more', 'time', 'we', 'spent', 'with', 'Gilad', 'as', 'well', 'as', 'the', 'rest', 'of', 'the', 'team', 'in', 'Israel', '(', 'not', 'to', 'mention', 'the', 'very', 'loud', '<unk>', 'that', 'runs', 'around', 'in', 'their', 'campus', ')', ',', 'the', 'more', 'convinced', 'we', 'all', 'became', '-', 'we', '’', 'll', 'be', 'better', 'off', 'together', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', '34', 'Diese', 'a', 'Worte', 'sind', 'wahr', 'und', 'treu', ';', 'darum', '<unk>', 'sie', 'nicht', ',', 'und', 'b', 'nehmt', 'auch', 'nichts', 'davon', 'weg', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', '34', 'These', 'sayings', 'are', 'a', 'true', 'and', 'faithful', ';', 'wherefore', ',', 'transgress', 'them', 'not', ',', 'neither', 'b', 'take', 'therefrom', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', '&#124;', 'Ferienwohnungen', '1', 'Zi', '&#124;', 'Ferienhäuser', '&#124;', 'Landhäuser', '&#124;', 'Autovermietung', '&#124;', 'Last', 'Minute', 'Angebote', '!', '!', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', '&#124;', '1', 'Bedroom', 'Apts', '&#124;', 'Holiday', 'houses', '&#124;', 'Rural', 'Homes', '&#124;', 'Car', 'Rental', '&#124;', 'Last', 'Minute', 'Offers', '!', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'Der', '<unk>', 'Teil', 'der', 'Insel', 'besteht', 'aus', 'Granit', 'und', '<unk>', ',', 'von', 'Ton', 'überlagert', ',', 'und', 'bildet', 'eine', 'ca', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'A', 'battle', 'between', 'Denmark', 'and', 'Sweden', 'in', '<unk>', 'led', 'to', 'Swedish', 'control', 'of', 'the', 'island', ',', 'but', 'it', 'was', 'brief', '-', 'they', 'left', 'again', 'the', 'same', 'year', '.', 'In', 'the', '<unk>', 'of', '<unk>', '<unk>', ',', '<unk>', ',', '<unk>', 'and', '<unk>', 'were', 'given', 'to', 'Sweden', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'Mag', 'sein', ',', 'dass', 'du', 'deine', 'ersten', '<unk>', 'in', 'einem', '<unk>', ',', '<unk>', 'Kahn', '<unk>', '-', 'aber', 'mit', 'der', 'Zeit', 'wirst', 'du', 'dich', 'zum', '<unk>', '<unk>', 'oder', 'edlen', 'Katamaran', '<unk>', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'You', 'may', 'be', 'starting', 'in', 'a', '<unk>', 'old', 'tub', 'of', 'a', 'boat', ',', 'but', 'in', 'no', 'time', 'at', 'all', 'you', '&apos;ll', 'be', 'able', 'to', 'buy', 'a', 'fancy', '<unk>', ',', 'or', 'a', 'classy', 'catamaran', '.', 'Turn', 'your', 'newfound', 'fame', 'into', 'money', ',', 'and', 'spend', 'it', 'to', 'buy', 'lavish', 'new', 'homes', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'In', 'der', '<unk>', 'im', 'Internet', 'müßte', 'die', 'Zufahrt', 'beschrieben', 'werden', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'There', 'are', 'no', 'adverse', 'comments', 'about', 'this', 'hotel', 'at', 'all', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'Ideale', 'Lage', 'für', 'Exkursionen', 'in', 'die', 'Stadt', 'und', 'Nähe', 'zur', 'Promenade', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'There', 'was', 'plenty', 'of', 'space', 'in', 'the', 'room', 'and', 'a', 'nice', 'garden', 'to', 'sit', 'and', 'have', 'a', 'drink', 'and', 'smoke', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'Das', 'Hotel', '<unk>', 'verfügt', 'über', 'eine', 'ideale', ',', 'ruhige', 'Lage', 'in', 'einem', 'geschäftigen', 'Viertel', 'mit', 'guter', 'Verkehrsanbindung', '.', 'Der', 'Bahnhof', 'und', 'eine', 'U', '##AT##-##AT##', 'Bahnstation', 'liegen', 'in', 'der', 'Nähe', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'Hotel', '<unk>', 'welcomes', 'you', 'to', 'a', 'busy', 'yet', 'quiet', 'area', 'of', 'Milan', ',', 'within', 'walking', 'distance', 'of', 'excellent', 'transport', 'links', ',', 'including', 'the', 'central', 'railway', 'station', 'and', 'the', 'Repubblica', 'metro', 'station', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'Zum', 'klimatisierten', 'Hotel', 'gehören', 'auch', 'ein', '<unk>', 'und', 'eine', 'traumhafte', 'Sonnenterrasse', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "\t ['<s>', 'Apart', 'from', 'this', ',', 'the', 'guests', 'can', 'enjoy', 'the', 'facility', 'of', 'an', 'independent', 'air', '##AT##-##AT##', 'conditioning', 'system', ',', 'a', 'jacuzzi', 'and', 'a', 'beautiful', 'sun', 'terrace', '.', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>']\n"
     ]
    }
   ],
   "source": [
    "# ================================================================================\n",
    "# Processing training data\n",
    "\n",
    "src_unk_count, tgt_unk_count = 0, 0\n",
    "\n",
    "train_inputs = []\n",
    "train_outputs = []\n",
    "\n",
    "# Chosen based on previously found statistics\n",
    "src_max_sent_length = 41 \n",
    "tgt_max_sent_length = 61\n",
    "\n",
    "print('Processing Training Data ...\\n')\n",
    "for s_i, (src_sent, tgt_sent) in enumerate(zip(source_sent,target_sent)):\n",
    "    # Break source and target sentences to word lists\n",
    "    src_sent_tokens = split_to_tokens(src_sent,True)\n",
    "    tgt_sent_tokens = split_to_tokens(tgt_sent,False)\n",
    "    \n",
    "    # Append <s> token's ID to the beggining of source sentence\n",
    "    num_src_sent = [src_dictionary['<s>']]\n",
    "    # Add the rest of word IDs for words found in the source sentence \n",
    "    for tok in src_sent_tokens:\n",
    "        if tok in src_dictionary.keys():\n",
    "            num_src_sent.append(src_dictionary[tok])\n",
    "\n",
    "    # If the lenghth of the source sentence below the maximum allowed length\n",
    "    # append </s> token's ID to the end\n",
    "    if len(num_src_sent)<src_max_sent_length:\n",
    "        num_src_sent.extend([src_dictionary['</s>'] for _ in range(src_max_sent_length - len(num_src_sent))])\n",
    "\n",
    "    # If the length exceed the maximum allowed length\n",
    "    # truncate the sentence\n",
    "    elif len(num_src_sent)>src_max_sent_length:\n",
    "        num_src_sent = num_src_sent[:src_max_sent_length]\n",
    "        \n",
    "    # Make sure the sentence is of length src_max_sent_length\n",
    "    assert len(num_src_sent)==src_max_sent_length,len(num_src_sent)\n",
    "\n",
    "    train_inputs.append(num_src_sent)\n",
    "    \n",
    "    # Create the numeric target sentence with word IDs\n",
    "    # append <s> to the beginning and append actual words later\n",
    "    num_tgt_sent = [tgt_dictionary['<s>']]\n",
    "    for tok in tgt_sent_tokens:\n",
    "        if tok in tgt_dictionary.keys():\n",
    "            num_tgt_sent.append(tgt_dictionary[tok])\n",
    "        \n",
    "    ## Modifying the outputs such that all the outputs have max_length elements\n",
    "    if len(num_tgt_sent)<tgt_max_sent_length:\n",
    "        num_tgt_sent.extend([tgt_dictionary['</s>'] for _ in range(tgt_max_sent_length - len(num_tgt_sent))])\n",
    "    elif len(num_tgt_sent)>tgt_max_sent_length:\n",
    "        num_tgt_sent = num_tgt_sent[:tgt_max_sent_length]\n",
    "        \n",
    "    train_outputs.append(num_tgt_sent)\n",
    "    \n",
    "print('Unk counts Src: %d, Tgt: %d'%(src_unk_count, tgt_unk_count))\n",
    "print('Sentences ',len(train_inputs))\n",
    "\n",
    "assert len(train_inputs)  == len(source_sent),\\\n",
    "        'Size of total elements: %d, Total sentences: %d'\\\n",
    "                %(len(train_inputs),len(source_sent))\n",
    "\n",
    "# Making inputs and outputs NumPy arrays\n",
    "train_inputs = np.array(train_inputs, dtype=np.int32)\n",
    "train_outputs = np.array(train_outputs, dtype=np.int32)\n",
    "\n",
    "# Make sure number of inputs and outputs dividable by 100\n",
    "train_inputs = train_inputs[:(train_inputs.shape[0]//100)*100,:]\n",
    "train_outputs = train_outputs[:(train_outputs.shape[0]//100)*100,:]\n",
    "print('\\t Done processing training data \\n')\n",
    "\n",
    "# Printing some data\n",
    "print('Samples from training data')\n",
    "for ti in range(10):\n",
    "    print('\\t',[src_reverse_dictionary[w]  for w in train_inputs[ti,:].tolist()])\n",
    "    print('\\t',[tgt_reverse_dictionary[w]  for w in train_outputs[ti,:].tolist()])\n",
    "print()\n",
    "print('\\tSentences ',train_inputs.shape[0])\n",
    "\n",
    "# ================================================================================\n",
    "# Processing Test data\n",
    "\n",
    "src_unk_count, tgt_unk_count = 0, 0\n",
    "print('Processing testing data ....\\n')\n",
    "test_inputs = []\n",
    "test_outputs = []\n",
    "for s_i, (src_sent,tgt_sent) in enumerate(zip(test_source_sent,test_target_sent)):\n",
    "    src_sent_tokens = split_to_tokens(src_sent,True)\n",
    "    tgt_sent_tokens = split_to_tokens(tgt_sent,False)\n",
    "    \n",
    "    num_src_sent = [src_dictionary['<s>']]\n",
    "    for tok in src_sent_tokens:\n",
    "        if tok in src_dictionary.keys():\n",
    "            num_src_sent.append(src_dictionary[tok])\n",
    "    \n",
    "    num_tgt_sent = [src_dictionary['<s>']]\n",
    "    for tok in tgt_sent_tokens:\n",
    "        if tok in tgt_dictionary.keys():\n",
    "            num_tgt_sent.append(tgt_dictionary[tok])\n",
    "        \n",
    "    # Append </s> if the length is not src_max_sent_length\n",
    "    if len(num_src_sent)<src_max_sent_length:\n",
    "        num_src_sent.extend([src_dictionary['</s>'] for _ in range(src_max_sent_length - len(num_src_sent))])\n",
    "    # Truncate the sentence if length is over src_max_sent_length\n",
    "    elif len(num_src_sent)>src_max_sent_length:\n",
    "        num_src_sent = num_src_sent[:src_max_sent_length]\n",
    "        \n",
    "    assert len(num_src_sent)==src_max_sent_length, len(num_src_sent)\n",
    "\n",
    "    test_inputs.append(num_src_sent)\n",
    "    \n",
    "    # Append </s> is length is not tgt_max_sent_length\n",
    "    if len(num_tgt_sent)<tgt_max_sent_length:\n",
    "        num_tgt_sent.extend([tgt_dictionary['</s>'] for _ in range(tgt_max_sent_length - len(num_tgt_sent))])\n",
    "    # Truncate the sentence if length over tgt_max_sent_length\n",
    "    elif len(num_tgt_sent)>tgt_max_sent_length:\n",
    "        num_tgt_sent = num_tgt_sent[:tgt_max_sent_length]\n",
    "        \n",
    "    assert len(num_tgt_sent)==tgt_max_sent_length, len(num_tgt_sent)\n",
    "\n",
    "    test_outputs.append(num_tgt_sent)\n",
    "\n",
    "# Printing some data\n",
    "print('Unk counts Tgt: %d, Tgt: %d'%(src_unk_count, tgt_unk_count))    \n",
    "print('Done processing testing data ....\\n')\n",
    "test_inputs = np.array(test_inputs,dtype=np.int32)\n",
    "test_outputs = np.array(test_outputs,dtype=np.int32)\n",
    "print('Samples from training data')\n",
    "for ti in range(10):\n",
    "    print('\\t',[src_reverse_dictionary[w]  for w in test_inputs[ti,:].tolist()])\n",
    "    print('\\t',[tgt_reverse_dictionary[w]  for w in test_outputs[ti,:].tolist()])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Flipping the Input Data\n",
    "Changin the order of the sentence of the target language improves the performance of NMT systems. Because when reversed, it helps the NMT system to establish a strong connection as the last word of the source language and the last word of the target language will be closest to each other. *DON'T RUN THIS MULTIPLE TIMES as running two times gives original.*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training and Test source data after flipping \n",
      "\t ['</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '.', 'können', 'nutzen', 'QuarkXPress', 'mit', 'zusammen', 'besten', 'am', '3', 'Suite', 'Creative', 'und', '2', 'Suite', 'Creative', 'Sie', 'wie', ',', 'Sie', 'erfahren', 'Hier', '<s>']\n",
      "\t ['tray', 'road', 'mistakes', 'of', 'expect', 'a', 'tabled', 'with', 'and', 'the', 'posts', 'useful', 'out', 'waiting', 'wounded', 'a', 'drinks', 'been', 'stand', '26th', 'and', 'senior', 'personal', ',', 'difficulties', 'qualifications', 'an', 'rather', 'road', 'rewriting', 'and', 'road', 'unsustainable', 'the', '2007', 'road', 'wounded', 'not', 'throughout', 'amendment', '<s>']\n",
      "\n",
      "\t ['</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '.', ')', 'Import', '##AT##-##AT##', 'PSD', '&gt;', 'Fenster', '(', 'Import', '##AT##-##AT##', 'PSD', 'Palette', 'die', 'Sie', 'öffnen', ',', 'können', 'zu', 'nutzen', 'Dateien', '##AT##-##AT##', 'PSD', 'von', 'Funktionen', 'speziellen', 'die', 'Um', '<s>']\n",
      "\t ['</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', '</s>', ',', '<unk>', 'and', 'important', '<unk>', 'important', 'the', '<unk>', '<unk>', 'the', 'CC', 'revolution', 'are', 'proposals', 'of', 'to', 'not', 'With', 'Overall', 'countries', 'more', '<s>']\n",
      "\n",
      "\n",
      "Testing data after flipping\n",
      "\t ['Creative', '®', 'Adobe', 'der', 'Anwendungen', 'von', 'unabhängig', 'das', 'und', ',', 'bedienen', 'Medien', 'alle', 'inzwischen', 'QuarkXPress', 'von', 'Anwender', 'können', '\\xad', 'CSS', 'und', 'HTML', 'Dank', '.', 'zuvor', 'jemals', 'als', 'besser', '®', 'Illustrator', 'und', '®', 'Photoshop', ',', '8', '®', 'QuarkXPress', 'sich', 'verstehen', 'Heute', '<s>']\n"
     ]
    }
   ],
   "source": [
    "## Reverse the Germen sentences\n",
    "# Remember reversing the source sentence gives better performance\n",
    "# DON'T RUN THIS MULTIPLE TIMES as running two times gives original\n",
    "train_inputs = np.fliplr(train_inputs)\n",
    "test_inputs = np.fliplr(test_inputs)\n",
    "\n",
    "print('Training and Test source data after flipping ')\n",
    "print('\\t',[src_reverse_dictionary[w] for w in train_inputs[0,:].tolist()])\n",
    "print('\\t',[tgt_reverse_dictionary[w] for w in test_inputs[0,:].tolist()])\n",
    "print()\n",
    "print('\\t',[src_reverse_dictionary[w] for w in train_inputs[10,:].tolist()])\n",
    "print('\\t',[tgt_reverse_dictionary[w] for w in test_inputs[10,:].tolist()])\n",
    "\n",
    "print()\n",
    "print('\\nTesting data after flipping')\n",
    "print('\\t',[src_reverse_dictionary[w] for w in test_inputs[0,:].tolist()])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Generations for MT\n",
    "\n",
    "Now we define the data generator for our NMT."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source data\n",
      "['</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "['</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "['</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "['</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "['</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "['</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "['</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "['</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "['</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "['</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "['</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "['</s>', '</s>', '.', '</s>', '</s>']\n",
      "['</s>', '</s>', 'bietet', '.', '</s>']\n",
      "['</s>', '</s>', 'Dateiformat', 'nutzen', '</s>']\n",
      "['</s>', '</s>', '##AT##-##AT##', 'optimal', '</s>']\n",
      "['</s>', '</s>', 'PSD', 'Bilder', '</s>']\n",
      "['</s>', '</s>', 'das', 'Ihre', '.']\n",
      "['</s>', '</s>', 'über', 'für', 'werden']\n",
      "['.', '</s>', 'Photoshop', 'es', 'ausgewählt']\n",
      "['können', '.', 'mit', 'Sie', 'Verwendungszweck']\n",
      "\n",
      "Target data batch\n",
      "['Here', 'QuarkXPress', 'In', 'For', 'If']\n",
      "[',', '8', 'this', 'example', 'you']\n",
      "['you', 'is', 'section', ',', 'use']\n",
      "['’', 'considered', 'we', 'you', 'PSD']\n",
      "['ll', 'by', '’', 'may', ',']\n",
      "['find', 'many', 'll', 'have', 'you']\n",
      "['out', 'to', 'explain', 'multiple', 'can']\n",
      "['how', 'have', 'when', 'layers', 'switch']\n",
      "['Creative', 'the', 'you', 'in', 'those']\n",
      "['Suite', 'best', 'should', 'your', 'layers']\n",
      "['users', 'integration', 'use', 'PSD', 'on']\n",
      "['can', 'with', 'the', 'with', 'or']\n",
      "['get', 'Photoshop', 'PSD', 'different', 'off']\n",
      "['the', '’', 'format', 'product', 'in']\n",
      "['best', 's', 'for', 'shots', 'QuarkXPress']\n",
      "['possible', 'PSD', 'your', ',', 'without']\n",
      "['interaction', 'file', 'images', 'which', 'having']\n",
      "['with', 'format', 'and', 'will', 'to']\n",
      "['QuarkXPress', 'of', 'how', 'vary', 'save']\n",
      "['.', 'any', 'to', 'from', 'a']\n",
      "['</s>', 'layout', 'get', 'publication', 'separate']\n",
      "['</s>', 'tool', 'the', 'to', 'TIFF']\n",
      "['</s>', 'available', 'most', 'publication', 'for']\n",
      "['</s>', 'today', 'out', '.', 'each']\n",
      "['</s>', '.', 'of', '</s>', 'publication']\n",
      "['</s>', '</s>', 'them', '</s>', '.']\n",
      "['</s>', '</s>', '.', '</s>', '</s>']\n",
      "['</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "['</s>', '</s>', '</s>', '</s>', '</s>']\n",
      "['</s>', '</s>', '</s>', '</s>', '</s>']\n"
     ]
    }
   ],
   "source": [
    "emb_mat = np.load('en-embeddings.npy')\n",
    "input_size = emb_mat.shape[1]\n",
    "\n",
    "class DataGeneratorMT(object):\n",
    "    \n",
    "    def __init__(self,batch_size,num_unroll,is_source, is_train):\n",
    "        global input_size\n",
    "        # Number of data points in a batch\n",
    "        self._batch_size = batch_size\n",
    "        # Number of unrollings\n",
    "        self._num_unroll = num_unroll\n",
    "        # Cursors for each element in batch\n",
    "        self._cursor = [0 for offset in range(self._batch_size)]\n",
    "        \n",
    "        # The sentence IDs being currently processed to create the \n",
    "        # current batch\n",
    "        self._sent_ids = None\n",
    "        \n",
    "        # We want a batch of data from source or target?\n",
    "        self._is_source = is_source\n",
    "        # Is this training or testing data?\n",
    "        self._is_train = is_train\n",
    "                \n",
    "    def next_batch(self, sent_ids):\n",
    "        \n",
    "        # Depending on wheter we want source or target data\n",
    "        # change the maximum sentence length\n",
    "        if self._is_source:\n",
    "            max_sent_length = src_max_sent_length\n",
    "        else:\n",
    "            max_sent_length = tgt_max_sent_length\n",
    "            \n",
    "        # Arrays to hold input and output data\n",
    "        batch_data = np.zeros((self._batch_size),dtype=np.float32)\n",
    "        batch_labels = np.zeros((self._batch_size),dtype=np.float32)\n",
    "        \n",
    "        \n",
    "        # Populate each index of the batch\n",
    "        for b in range(self._batch_size):\n",
    "            \n",
    "            # Sentence IDs to get data from\n",
    "            sent_id = sent_ids[b]\n",
    "            \n",
    "            # If generating data with source sentences\n",
    "            # use src_word_embeddings\n",
    "            if self._is_source:\n",
    "                # Depending on whether we need training data or testind data\n",
    "                # choose the previously created training or testind data\n",
    "                if self._is_train:\n",
    "                    sent_text = train_inputs[sent_id]\n",
    "                else:\n",
    "                    sent_text = test_inputs[sent_id]\n",
    "                             \n",
    "                # Populate the batch data arrays\n",
    "                batch_data[b] = sent_text[self._cursor[b]]\n",
    "                batch_labels[b] = sent_text[self._cursor[b]+1]\n",
    "            # If generating data with target sentences\n",
    "            # use tgt_word_embeddings\n",
    "            else:\n",
    "                # Depending on whether we need training data or testind data\n",
    "                # choose the previously created training or testind data\n",
    "                if self._is_train:\n",
    "                    sent_text = train_outputs[sent_id]\n",
    "                else:\n",
    "                    sent_text = test_outputs[sent_id]\n",
    "                \n",
    "                # We cannot avoid having two different embedding vectors for <s> token\n",
    "                # in soruce and target languages\n",
    "                # Therefore, if the symbol appears, we always take the source embedding vector\n",
    "                if sent_text[self._cursor[b]]!=tgt_dictionary['<s>']:\n",
    "                    batch_data[b] = sent_text[self._cursor[b]]\n",
    "                else:\n",
    "                    batch_data[b] = sent_text[self._cursor[b]]\n",
    "                \n",
    "                # Populate the data arrays\n",
    "                batch_labels[b] = sent_text[self._cursor[b]+1]\n",
    "            \n",
    "            # Update the cursor for each batch index\n",
    "            self._cursor[b] = (self._cursor[b]+1)%(max_sent_length-1)\n",
    "             \n",
    "        return batch_data,batch_labels\n",
    "        \n",
    "    def unroll_batches(self,sent_ids):\n",
    "        \n",
    "        # Only if new sentence IDs if provided\n",
    "        # else it will use the previously defined \n",
    "        # sent_ids continuously\n",
    "        if sent_ids is not None:\n",
    "            \n",
    "            self._sent_ids = sent_ids\n",
    "            # Unlike in the previous exercises we do not process a single sequence\n",
    "            # over many iterations of unrollings. We process either a source sentence or target sentence\n",
    "            # at a single go. So we reset the _cursor evrytime we generate a batch\n",
    "            self._cursor = [0 for _ in range(self._batch_size)]\n",
    "                \n",
    "        unroll_data,unroll_labels = [],[]\n",
    "        \n",
    "        # Unrolling data over time\n",
    "        for ui in range(self._num_unroll):\n",
    "            \n",
    "            if self._is_source:\n",
    "                data, labels = self.next_batch(self._sent_ids)\n",
    "            else:\n",
    "                data, labels = self.next_batch(self._sent_ids)\n",
    "                    \n",
    "            unroll_data.append(data)\n",
    "            unroll_labels.append(labels)\n",
    "        \n",
    "        # Return unrolled data and sentence IDs\n",
    "        return unroll_data, unroll_labels, self._sent_ids\n",
    "    \n",
    "    def reset_indices(self):\n",
    "        self._cursor = [0 for offset in range(self._batch_size)]\n",
    "        \n",
    "# Running a tiny set to see if the implementation correct\n",
    "dg = DataGeneratorMT(batch_size=5,num_unroll=20,is_source=True, is_train=True)\n",
    "u_data, u_labels, _ = dg.unroll_batches([0,1,2,3,4])\n",
    "\n",
    "print('Source data')\n",
    "for _, lbl in zip(u_data,u_labels):\n",
    "    # the the string words for returned word IDs and display the results\n",
    "    print([src_reverse_dictionary[w] for w in lbl.tolist()])\n",
    "\n",
    "        # Running a tiny set to see if the implementation correct\n",
    "dg = DataGeneratorMT(batch_size=5,num_unroll=30,is_source=False, is_train=True)\n",
    "u_data, u_labels, _ = dg.unroll_batches([0,2,3,4,5])\n",
    "print('\\nTarget data batch')\n",
    "for d_i,(_, lbl) in enumerate(zip(u_data,u_labels)):\n",
    "    # the the string words for returned word IDs and display the results\n",
    "    print([tgt_reverse_dictionary[w] for w in lbl.tolist()])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NMT using Tensorflow seq2seq library\n",
    "\n",
    "## Defining the TensorFlow inputs and outputs\n",
    "Here we define placeholder to carry inputs and outputs that are required optimize the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "enc_train_inputs = []\n",
    "dec_train_inputs, dec_train_labels = [],[]\n",
    "\n",
    "# Need to use pre-trained word embeddings\n",
    "encoder_emb_layer = tf.convert_to_tensor(np.load('de-embeddings.npy'))\n",
    "decoder_emb_layer = tf.convert_to_tensor(np.load('en-embeddings.npy'))\n",
    "\n",
    "# Defining unrolled training inputs\n",
    "for ui in range(source_sequence_length):\n",
    "    enc_train_inputs.append(tf.placeholder(tf.int32, shape=[batch_size],name='train_inputs_%d'%ui))\n",
    "\n",
    "# Define unrolled training outputs\n",
    "for ui in range(target_sequence_length):\n",
    "    dec_train_inputs.append(tf.placeholder(tf.int32, shape=[batch_size],name='train_inputs_%d'%ui))\n",
    "    dec_train_labels.append(tf.placeholder(tf.int32, shape=[batch_size],name='train_outputs_%d'%ui))\n",
    "\n",
    "# Define embedding lookup operation\n",
    "encoder_emb_inp = [tf.nn.embedding_lookup(encoder_emb_layer, src) for src in enc_train_inputs]\n",
    "encoder_emb_inp = tf.stack(encoder_emb_inp)\n",
    "\n",
    "decoder_emb_inp = [tf.nn.embedding_lookup(decoder_emb_layer, src) for src in dec_train_inputs]\n",
    "decoder_emb_inp = tf.stack(decoder_emb_inp)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define the Encoder\n",
    "\n",
    "Now we define the encoder cell. Encoder is a simple LSTM cell provided in seq2seq library as `BasicLSTMCell`. Then we use the `dynamic_rnn`function to unroll our inputs and get the output cell state."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "encoder_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units)\n",
    "\n",
    "initial_state = encoder_cell.zero_state(batch_size, dtype=tf.float32)\n",
    "\n",
    "encoder_outputs, encoder_state = tf.nn.dynamic_rnn(\n",
    "    encoder_cell, encoder_emb_inp, initial_state=initial_state,\n",
    "    sequence_length=[source_sequence_length for _ in range(batch_size)], \n",
    "    time_major=True, swap_memory=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define the Encoder\n",
    "We now define the decoder cell and an output softmax layer (`projection_layer`) as well as a helper (that produce word embeddings). Note that we give the reader the option to change the type of decoder (that is, with or without attention)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Build RNN cell\n",
    "decoder_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units)\n",
    "\n",
    "projection_layer = Dense(units=vocab_size, use_bias=True)\n",
    "\n",
    "# Helper\n",
    "helper = tf.contrib.seq2seq.TrainingHelper(\n",
    "    decoder_emb_inp, [target_sequence_length for _ in range(batch_size)], time_major=True)\n",
    "\n",
    "# Decoder\n",
    "if decoder_type == 'basic':\n",
    "    decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "        decoder_cell, helper, encoder_state,\n",
    "        output_layer=projection_layer)\n",
    "    \n",
    "elif decoder_type == 'attention':\n",
    "    decoder = tf.contrib.seq2seq.BahdanauAttention(\n",
    "        decoder_cell, helper, encoder_state,\n",
    "        output_layer=projection_layer)\n",
    "    \n",
    "# Dynamic decoding\n",
    "outputs, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "    decoder, output_time_major=True,\n",
    "    swap_memory=True\n",
    ")\n",
    "\n",
    "\n",
    "# Computing logits and predictions\n",
    "logits = outputs.rnn_output\n",
    "train_prediction = outputs.sample_id\n",
    "\n",
    "# Loss computation\n",
    "crossent = tf.nn.sparse_softmax_cross_entropy_with_logits(\n",
    "    labels=dec_train_labels, logits=logits)\n",
    "loss = tf.reduce_mean(crossent) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining optimizer\n",
    "Here we define the optimizers to optimize the model parameters. As shown before, we use two optimizers Adam and SGD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defining Optimizer\n"
     ]
    }
   ],
   "source": [
    "print('Defining Optimizer')\n",
    "# Adam Optimizer. And gradient clipping.\n",
    "global_step = tf.Variable(0, trainable=False)\n",
    "inc_gstep = tf.assign(global_step,global_step + 1)\n",
    "learning_rate = tf.train.exponential_decay(\n",
    "    0.01, global_step, decay_steps=10, decay_rate=0.9, staircase=True)\n",
    "with tf.variable_scope('Adam'):\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "with tf.variable_scope('SGD'):\n",
    "    sgd_optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "#optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "gradients, v = zip(*optimizer.compute_gradients(loss))\n",
    "gradients, _ = tf.clip_by_global_norm(gradients, 25.0)\n",
    "optimize = optimizer.apply_gradients(zip(gradients, v))\n",
    "\n",
    "sgd_gradients, v = zip(*sgd_optimizer.compute_gradients(loss))\n",
    "sgd_gradients, _ = tf.clip_by_global_norm(sgd_gradients, 25.0)\n",
    "sgd_optimize = optimizer.apply_gradients(zip(sgd_gradients, v))\n",
    "\n",
    "sess = tf.InteractiveSession()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running the Seq2seq NMT\n",
    "\n",
    "We now run the seq2seq NMT system we defined."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Started Training\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "Step  500\n",
      "Actual: the program <unk> ( Windows only ) helps Braille Music <unk> . </s> \n",
      "\n",
      "Predicted: The hotel is is <unk> <unk> the , be <unk> . . </s> \n",
      "\n",
      "\n",
      "Actual: Driving during morning and afternoon peak hours is not recommended , as traffic slows to a standstill and even a simple trip across a bridge can take up to 45 minutes . </s> \n",
      "\n",
      "Predicted: The <unk> the , the . . . a a to and the , . the <unk> . the to <unk> of . the <unk> . be the to the . . </s> \n",
      "\n",
      "============= Step  500  =============\n",
      "\t Loss:  2.907963124513626\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "Step  1000\n",
      "Actual: Enjoying sun , sea and juices in the restaurants with plank beds , which you can use free of charge . </s> \n",
      "\n",
      "Predicted: The a , the and the , the heart , a <unk> . a are can enjoy the of the . </s> \n",
      "\n",
      "\n",
      "Actual: Roddenberry also receives credit for writing lyrics to the TOS main title theme , although these lyrics were never recorded in connection with the series . </s> \n",
      "\n",
      "Predicted: The <unk> <unk> the , the , , be <unk> of of , . and you are . not . . the . the <unk> of </s> \n",
      "\n",
      "============= Step  1000  =============\n",
      "\t Loss:  2.431515670537949\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "Step  1500\n",
      "Actual: Next to many shops , bars , restaurants and nightclubs as Es <unk> and Eden . </s> \n",
      "\n",
      "Predicted: The to the of , the , and , the , well <unk> , <unk> , </s> \n",
      "\n",
      "\n",
      "Actual: The hotel is also convenient for shoppers , with the Plaza Norte 2 and <unk> shopping centres just a short distance away . </s> \n",
      "\n",
      "Predicted: The hotel is located a to the , and the <unk> <unk> and ##AT##-##AT## the <unk> , . . few walk of from </s> \n",
      "\n",
      "============= Step  1500  =============\n",
      "\t Loss:  2.307385681152344\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "Step  2000\n",
      "Actual: Restaurant , Bar , 24 ##AT##-##AT## Hour Front Desk , Newspapers , Terrace , Non ##AT##-##AT## Smoking Rooms , Rooms / Facilities for Disabled Guests , Elevator , Express Check ##AT##-##AT## In / Check ##AT##-##AT## Out , Safety Deposit Box , Soundproofed Rooms , Heating , Luggage Storage , Shops in Hotel , Gay Friendly , All Public and Private \n",
      "\n",
      "Predicted: The , Bar , 24 ##AT##-##AT## Hour Front Desk , Newspapers , Non , Non ##AT##-##AT## Smoking Rooms , Safety / Facilities for Disabled Guests , Elevator , Safety Check ##AT##-##AT## In / Check ##AT##-##AT## Out , Safety Deposit Box , Soundproofed Rooms , Tour , Luggage Storage , Tour in Hotel . Gay Friendly , Car Public and Private \n",
      "\n",
      "\n",
      "Actual: Este un hotel care <unk> <unk> de <unk> la <unk> <unk> , <unk> au <unk> <unk> , <unk> nu era <unk> <unk> <unk> <unk> <unk> <unk> <unk> . </s> \n",
      "\n",
      "Predicted: The <unk> <unk> <unk> , , <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> </s> \n",
      "\n",
      "============= Step  2000  =============\n",
      "\t Loss:  2.2107269892692565\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "Step  2500\n",
      "Actual: Consumer research has shown that demand is growing for <unk> with wide ##AT##-##AT## angle lenses . A 28mm lens allows users to capture large groups of people indoors or wide shots of buildings . </s> \n",
      "\n",
      "Predicted: The <unk> and been to the , the , the , the ##AT##-##AT## <unk> , . </s> \n",
      "\n",
      "\n",
      "Actual: It is hard to imagine the developer who is using , e <unk> . , different <unk> on each HTML page or who is using the groups of ten similar pictures with different names for each HTML page . </s> \n",
      "\n",
      "Predicted: The is a to be the <unk> of are not the and ##AT##-##AT## , </s> \n",
      "\n",
      "============= Step  2500  =============\n",
      "\t Loss:  2.230203649520874\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "Step  3000\n",
      "Actual: Capri is a Mediterranean island of calcareous origin that has been visited over the centuries by intellectuals , artists and writers , all enthralled by its magical beauty . A mix of history , nature , <unk> , culture and events that daily blend together and bring the Legend of Capri to life ; a legend that sees no comparison \n",
      "\n",
      "Predicted: The , a new ##AT##-##AT## of the <unk> of is a a by the world of the , the , the of and the the the <unk> and . </s> \n",
      "\n",
      "\n",
      "Actual: In the park live four different kinds of <unk> : the European wolf , the Timber wolf , the <unk> Wolf and the white <unk> from Alaska ( Polar Wolf ) . </s> \n",
      "\n",
      "Predicted: The the case , , ##AT##-##AT## types of the , <unk> <unk> Union , and <unk> <unk> and and <unk> of , the <unk> <unk> of the . <unk> ) ) . </s> \n",
      "\n",
      "============= Step  3000  =============\n",
      "\t Loss:  2.144723793506622\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "Step  3500\n",
      "Actual: Actually located at the <unk> , all kinds of shops and <unk> <unk> ( mobile cafes selling <unk> wine - delicious ! ) now stretch through the whole city center during this period . </s> \n",
      "\n",
      "Predicted: <unk> , in the hotel <unk> <unk> the of the , restaurants . . <unk> ) ) ) ) ) <unk> and </s> \n",
      "\n",
      "\n",
      "Actual: The staff was very accomodating both with our needs with regard to separate beds as well as breakfast items . </s> \n",
      "\n",
      "Predicted: <unk> hotel were very helpful and , the room and the to the and . well as the . . </s> \n",
      "\n",
      "============= Step  3500  =============\n",
      "\t Loss:  2.132036427021027\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "Step  4000\n",
      "Actual: Location was excellent for walking to landmarks and monuments . The hotel staff was very friendly and accommodating . </s> \n",
      "\n",
      "Predicted: The is very and the distance the and the . </s> \n",
      "\n",
      "\n",
      "Actual: <unk> : User en ##AT##-##AT## 1 – <unk> <unk> <unk> <unk> , <unk> so <unk> . </s> \n",
      "\n",
      "Predicted: <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> </s> \n",
      "\n",
      "============= Step  4000  =============\n",
      "\t Loss:  2.0885994465351105\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "Step  4500\n",
      "Actual: The Heroic dungeon daily quest will now reward 2 <unk> of Triumph and the normal daily dungeon quest will reward 1 <unk> of Triumph . </s> \n",
      "\n",
      "Predicted: The <unk> of of is for be be the <unk> <unk> the , <unk> <unk> <unk> is in . be the <unk> <unk> the , </s> \n",
      "\n",
      "\n",
      "Actual: <unk> exchanges AB records directly between OS X users , even multiple users logged on <unk> on a single system . </s> \n",
      "\n",
      "Predicted: <unk> is and ##AT##-##AT## are to the and <unk> , and more applications , in the . the single server . </s> \n",
      "\n",
      "============= Step  4500  =============\n",
      "\t Loss:  2.0541179225444792\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "Step  5000\n",
      "Actual: 29 And the Gods said : Behold , we will give them every herb bearing seed that shall come upon the face of all the earth , and every tree which shall have fruit upon it ; yea , the fruit of the tree yielding seed to them we will give it ; it shall be for their a meat \n",
      "\n",
      "Predicted: 13 And it Lord shall unto a , I shall be you in day , , , ye be to the Holy of the the words , and the man , shall be a of the , and , and Lord of the Lord of the . the . shall be their . and shall be a the own <unk> \n",
      "\n",
      "\n",
      "Actual: Now , application quality and IT efficiency are seen as major business components and <unk> . </s> \n",
      "\n",
      "Predicted: The , the is and <unk> ##AT##-##AT## , available in a and and and the , </s> \n",
      "\n",
      "============= Step  5000  =============\n",
      "\t Loss:  2.033629116296768\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "Step  5500\n",
      "Actual: We also give you the opportunity to opt ##AT##-##AT## out and / or modify information you have provided to us at any time by logging on to the applicable section ( for example , newsletter subscriptions or &quot; my account &quot; ) and selecting those portions of your personal information that you would like to update . </s> \n",
      "\n",
      "Predicted: We are have you to right to use to to to the or a the , need to a your , the time . clicking , your your Internet or , or example , or or ) ) <unk> &quot; &quot; ) . you the of . the computer information . you want like to use . </s> \n",
      "\n",
      "\n",
      "Actual: Has Europe lost its humanist ideal ? </s> \n",
      "\n",
      "Predicted: What a in in own ? ? </s> \n",
      "\n",
      "============= Step  5500  =============\n",
      "\t Loss:  2.014813478708267\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "....................................................................................................\n",
      "........................................................................."
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-25-5257cd195502>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     23\u001b[0m     \u001b[1;31m# num_enc_unrollings: 40\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     24\u001b[0m     \u001b[1;31m# num_dec_unrollings: 60\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 25\u001b[1;33m     \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'.'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mend\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m''\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     26\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m%\u001b[0m\u001b[1;36m100\u001b[0m\u001b[1;33m==\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     27\u001b[0m         \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m''\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\ipykernel\\iostream.py\u001b[0m in \u001b[0;36mwrite\u001b[1;34m(self, string)\u001b[0m\n\u001b[0;32m    350\u001b[0m             \u001b[0mis_child\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_is_master_process\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    351\u001b[0m             \u001b[1;31m# only touch the buffer in the IO thread to avoid races\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 352\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpub_thread\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mschedule\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;32mlambda\u001b[0m \u001b[1;33m:\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_buffer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstring\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    353\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mis_child\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    354\u001b[0m                 \u001b[1;31m# newlines imply flush in subprocesses\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\ipykernel\\iostream.py\u001b[0m in \u001b[0;36mschedule\u001b[1;34m(self, f)\u001b[0m\n\u001b[0;32m    188\u001b[0m                 \u001b[0mevent_id\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0murandom\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m16\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    189\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_events\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mevent_id\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 190\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_event_pipe\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mevent_id\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    191\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    192\u001b[0m             \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mzmq\\backend\\cython\\socket.pyx\u001b[0m in \u001b[0;36mzmq.backend.cython.socket.Socket.send (zmq\\backend\\cython\\socket.c:7305)\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mzmq\\backend\\cython\\socket.pyx\u001b[0m in \u001b[0;36mzmq.backend.cython.socket.Socket.send (zmq\\backend\\cython\\socket.c:7048)\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mzmq\\backend\\cython\\socket.pyx\u001b[0m in \u001b[0;36mzmq.backend.cython.socket._send_copy (zmq\\backend\\cython\\socket.c:2920)\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mc:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\zmq\\backend\\cython\\checkrc.pxd\u001b[0m in \u001b[0;36mzmq.backend.cython.checkrc._check_rc (zmq\\backend\\cython\\socket.c:9621)\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "loss_over_time = []\n",
    "\n",
    "# Initialize TensorFlow variables\n",
    "tf.global_variables_initializer().run()\n",
    "\n",
    "# Load the word embeddings\n",
    "src_word_embeddings = np.load('de-embeddings.npy')\n",
    "tgt_word_embeddings = np.load('en-embeddings.npy')\n",
    "\n",
    "# Defining data generators\n",
    "enc_data_generator = DataGeneratorMT(\n",
    "    batch_size=batch_size,num_unroll=source_sequence_length,is_train=True, is_source=True)\n",
    "dec_data_generator = DataGeneratorMT(\n",
    "    batch_size=batch_size,num_unroll=target_sequence_length,is_train=True, is_source=False)\n",
    "\n",
    "num_steps = 10001\n",
    "avg_loss = 0\n",
    "\n",
    "print('Started Training')\n",
    "\n",
    "for step in range(num_steps):\n",
    "\n",
    "    # num_enc_unrollings: 40\n",
    "    # num_dec_unrollings: 60\n",
    "    print('.',end='')\n",
    "    if (step+1)%100==0:\n",
    "        print('')\n",
    "        \n",
    "    # Pick a random batch of sentences to train the algorithm\n",
    "    sent_ids = np.random.randint(low=0,high=train_inputs.shape[0],size=(batch_size))\n",
    "\n",
    "    # Create a batch of data for the encoder\n",
    "    eu_data, eu_labels, _ = enc_data_generator.unroll_batches(sent_ids=sent_ids)\n",
    "    \n",
    "    # Create a batch of data for the decoder\n",
    "    du_data, du_labels, _ = dec_data_generator.unroll_batches(sent_ids=sent_ids)\n",
    "    \n",
    "    feed_dict = {}\n",
    "    for ui,(dat,lbl) in enumerate(zip(eu_data,eu_labels)):            \n",
    "        feed_dict[enc_train_inputs[ui]] = dat                \n",
    "    \n",
    "    for ui,(dat,lbl) in enumerate(zip(du_data,du_labels)):            \n",
    "        feed_dict[dec_train_inputs[ui]] = dat\n",
    "        feed_dict[dec_train_labels[ui]] = lbl\n",
    "\n",
    "    # Optimize the NMT with either Adam (first 10000 iterations)\n",
    "    # or stochastic gradient descent (after 10000 iterations)\n",
    "    if (step+1)<10000:\n",
    "        _,l,tr_pred = sess.run([optimize,loss,train_prediction], feed_dict=feed_dict)\n",
    "        tr_pred = tr_pred.flatten()\n",
    "    else:\n",
    "        _,l,tr_pred = sess.run([sgd_optimize,loss,train_prediction], feed_dict=feed_dict)\n",
    "        tr_pred = tr_pred.flatten()\n",
    "        \n",
    "    # Print some training predictions\n",
    "    if (step+1)%100==0:  \n",
    "        \n",
    "        print('Step ',step+1)\n",
    "\n",
    "        # Print the train results (actual and predicted)\n",
    "        print_str = 'Actual: '\n",
    "        for w in np.concatenate(du_labels,axis=0)[::batch_size].tolist():\n",
    "            print_str += tgt_reverse_dictionary[w] + ' '                    \n",
    "            if tgt_reverse_dictionary[w] == '</s>':\n",
    "                break\n",
    "                      \n",
    "        print(print_str)\n",
    "        print()\n",
    "        \n",
    "        print_str = 'Predicted: '\n",
    "        for w in tr_pred[::batch_size].tolist():\n",
    "            print_str += tgt_reverse_dictionary[w] + ' '\n",
    "            if tgt_reverse_dictionary[w] == '</s>':\n",
    "                break\n",
    "        print(print_str)\n",
    "       \n",
    "        print('\\n')  \n",
    "        \n",
    "        rand_idx = np.random.randint(low=1,high=batch_size)\n",
    "        print_str = 'Actual: '\n",
    "        for w in np.concatenate(du_labels,axis=0)[rand_idx::batch_size].tolist():\n",
    "            print_str += tgt_reverse_dictionary[w] + ' '\n",
    "            if tgt_reverse_dictionary[w] == '</s>':\n",
    "                break\n",
    "        print(print_str)\n",
    "\n",
    "            \n",
    "        print()\n",
    "        print_str = 'Predicted: '\n",
    "        for w in tr_pred[rand_idx::batch_size].tolist():\n",
    "            print_str += tgt_reverse_dictionary[w] + ' '\n",
    "            if tgt_reverse_dictionary[w] == '</s>':\n",
    "                break\n",
    "        print(print_str)\n",
    "        print()        \n",
    "        \n",
    "    avg_loss += l # Update average loss\n",
    "    \n",
    "    # Print the loss\n",
    "    if (step+1)%500==0:\n",
    "        print('============= Step ', str(step+1), ' =============')\n",
    "        print('\\t Loss: ',avg_loss/500.0)\n",
    "        \n",
    "        loss_over_time.append(avg_loss/500.0)\n",
    "             \n",
    "        avg_loss = 0.0\n",
    "        sess.run(inc_gstep)\n",
    "            \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
